package com.example;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class RamPageSpider {

  static String cookieStr = "mywork.tab.tasks=false; _ga_VPBMX0QP83=deleted; _ga_VPBMX0QP83=deleted; mo.confluence-oauth.FORM_COOKIE=loginform; mo.confluence-oauth.LOGOUTCOOKIE=LogoutCookie; _gid=GA1.2.375705832.1722819981; space_auth_live=MTcyMjgyOTU5MXxOd3dBTkU5Q01rNVhSa3MwUjFKSk56VlNOVTFGUWpVMFNFbEdXbGszV0RWTk1qVk1Sa3N5V1VOVVNFNVVVVTFEVmxSV1dsTlJOVkU9fKp643TpqkZccsWamO0cMDDjAYvxOuQj3nNS25Twq-G1; _ga_N1FF831FCD=GS1.1.1722927919.8.0.1722927919.0.0.0; G_AUTHUSER_H=0; _ga_9HJTZDHFDL=GS1.1.1722942105.5.1.1722943296.0.0.0; _ga_V8CYP9D8PH=GS1.1.1722942152.2.1.1722943298.0.0.0; JSESSIONID=DE317123A483BC16A6A7CCDF81E61901; _ga_VPBMX0QP83=GS1.1.1723083694.476.1.1723083775.59.0.0; _ga=GA1.1.1865479565.1710398786";
  private static Map<String, String> cookies = new HashMap<String, String>() {{
    put("Cookie", "JSESSIONID=JSESSIONID");
  }};

  static {
    String[] cookie = cookieStr.replace(" ", "").split(";");
    for (String c : cookie) {
      cookies.put(c.split("=")[0], c.split("=")[1]);
    }
    System.out.println(cookies.toString());
  }

  private static final String DOMAIN = "https://confluence.shopee.io";

  public static void main(String[] args) throws IOException {
    Set<String> words = new HashSet<>();
    String url = "https://confluence.shopee.io/pages/viewpage.action?pageId=156918814";

    List<String> uris = findLinksOnHomePage(url, words);
    uris = uris.stream()
               .filter(uri -> uri.startsWith("/display") || uri.startsWith("/pages"))
               .filter(uri -> !uri.contains("removepage.action"))
               .distinct()
               .collect(Collectors.toList());
    System.out.println("---------------遍历子页面----------------");
    for (String uri : uris) {
      System.out.println("request child page: " + DOMAIN + uri);
      Document doc = Jsoup.connect(DOMAIN + uri).cookies(cookies).timeout(300000000).get();
      words.addAll(fetchUniqueWords(doc));
    }
    System.out.println("---------------------------------------------------");
    words.stream().filter(e -> e.length() > 2).sorted().forEach(System.out::println);
  }

  private static List<String> findLinksOnHomePage(String url, Set<String> words) throws IOException {
    List<String> result = new ArrayList<>();
    Document doc = Jsoup.connect(url).cookies(cookies).timeout(300000000).get();
    // 抓取所有的<a>标签并提取链接
    Elements links = doc.select("div#main-content a[href]");
    for (Element link : links) {
      // 提取并输出链接的 href 属性和文本
      String linkHref = link.attr("href");
      String linkText = link.text();
      result.add(linkHref);
      System.out.println(" Text: " + linkText + "         " + "Link: " + linkHref);
    }
    words.addAll(fetchUniqueWords(doc));
    return result;
  }


  private static Set<String> fetchUniqueWords(Document doc) {
    // 获取并解析HTML内容
    String text = doc.text();  // 提取网页中的文本内容
    System.out.println("----------------start----------------");
    System.out.println(text);
    System.out.println("----------------end----------------");
    // 使用正则表达式匹配英文单词和短语
    Pattern pattern = Pattern.compile("\\b[a-zA-Z]+(?:'[a-zA-Z]+)?\\b");
    Matcher matcher = pattern.matcher(text);
    Set<String> words = new HashSet<>();

    while (matcher.find()) {
      String word = matcher.group().toLowerCase();  // 将单词转换为小写以去重
      words.add(word);
    }
    return words;
  }
}
